# -*- coding: utf-8 -*-
# Edit : Dinh Manh Dau <dinhmanhdau@gmail.com>
import mechanize
import workerpool
import re, datetime, time, sys, os
import urlparse
import traceback
import pycommonlib as pyclib
import cStringIO as StringIO
from lxml import etree
from urllib import urlretrieve
os.umask(0000)

# <!--- basic function --
def freeMemory(item):
    if item is None:
        return
    try:
        item.close()
    except: pass
    item = None
    del item

def buildTree(url, outputHTML=False, returnHTML=False):
    tree = None
    maxRetries = 3
    currentRetry = 0
    flag = False
    errMsg = ''
    c_html = ''
    user_agent = 'Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10'
    while not flag and currentRetry < maxRetries:
        http = None
        response = None
        try:
            http = mechanize.build_opener(mechanize.HTTPRefererProcessor, mechanize.HTTPErrorProcessor)
            http.addheaders = [("User-agent", user_agent)]
            response = http.open(url)
            responseURL = response.geturl()
            html = None
            if responseURL != 'http://m.xalo.vn/tintuc.mobi':
                html = response.read()
                c_html = html
                if outputHTML: print html
                tree = etree.parse(StringIO.StringIO(html), etree.HTMLParser(encoding='utf-8'))
                flag = True
                html = None
        except:
            currentRetry += 1
            errMsg = "Error at ({0}, {1}:{2})".format(url, sys.exc_info()[0], sys.exc_info()[1])
        finally:
            freeMemory(http)
            freeMemory(response)
    if errMsg != '':
        print errMsg
        errMsg = None
    if returnHTML: return tree, c_html
    return tree

def getElementText(elem):
    str = ''
    if type(elem) == types.ListType and len(elem) > 0: elem = elem[0]
    if type(elem) == lxml.etree._Element:
        str = re.sub(r"\s+", ' ', lxml.html.fromstring(etree.tostring(elem)).text_content().strip())
    return str

def getAttrib(elem, attrib):
    str = ''
    if type(elem) == types.ListType and len(elem) > 0: elem = elem[0]
    if type(elem) == lxml.etree._Element:
        str = elem.get(attrib)
    return str.strip()

def saveFile(url):
    try:
      global slideshowTitle, __ROOT__
      folder = re.sub(r"\s+", ' ', re.sub(r"[^a-zA-Z0-9-\s]", '', re.sub(r"(\\u.+?;)", '', slideshowTitle)))
      filePath = "{0}{1}/{2}".format(__ROOT__, folder, reg_extract(r"(slide-\d+.jpg)", url, 1))
      maxRetries = 3
      currentRetry = 0
      flag = False
      while not flag and currentRetry < maxRetries:
          try:
              if not os.path.isdir(__ROOT__ + folder):    os.makedirs(__ROOT__ + folder)
          except: pass
          if not os.path.isfile(filePath):
              print ">> start download {0}".format(url)
              fileName, headers = urlretrieve(url, filePath)
              fileSize = int(reg_extract(r"Content-Length: (\d+)", str(headers), 1))
              if fileSize > 0:   
                  print ">> file {0} is downloaded successful".format(fileName)
                  flag = True
              else:   os.unlink(filePath)
          else: print ">>> FILE: {0} is exists.".format(filePath)
          currentRetry += 1
    except:
      traceback.print_exc()
     
def reg_extract(pat, mat, id):
    preg = re.compile(pat)
    m = preg.search(mat)
    if m != '':
        if (id>0):  return m.group(id)
        else:   return m.group()
    # -- end func -->
    
def process(url):
    print ">> PROCESSING: ", url, '\n'
    global slideshowTitle, refer, __ROOT__
    try:
#        if not re.search(r"/mobile/", url):
#            url = re.sub("slideshare.net", "slideshare.net/mobile", url)
        refer = url
        slideObj = ''
        tree, html = buildTree(url, returnHTML=True)
        listNode = tree.xpath('//script')
        for node in listNode: 
            text    = pyclib.getStringWithNode(node)
            m       = pyclib.regexString('var slideshare_mobile_object = (.+);', text)
            if m: slideObj =  m.group(1); break
        if len(slideObj)>1:
            slideshowTitle = reg_extract(r"\"slideshowTitle\":\"(.+?)\"", slideObj, 1)
            baseSlideUrl = reg_extract(r"\"baseSlideUrl\":\"(.+?)\"", slideObj, 1)
            totalSlides = int(reg_extract(r"\"totalSlides\":(\d+)", slideObj, 1))
            # check folder exsits
#            folder = re.sub(r"\s+", ' ', re.sub(r"[^a-zA-Z0-9-\s]", '', re.sub(r"(\\u.+?;)", '', slideshowTitle)))
#            if os.path.isdir(__ROOT__ + folder): 
#              print 'Folder is exists.'
#              return
            images = []
            for i in range(1, totalSlides+1):
                images.append('{0}-slide-{1}.jpg&big=1'.format(baseSlideUrl, i))
            pool = workerpool.WorkerPool(size=8)
            pool.map(saveFile, images)
            pool.shutdown(); pool.wait()
            print '>> finishes download images!!'
        else: 
            print '>> Could not get content.'

#        print etree.tostring(tree)
#        print len(tree.xpath("//script[contains(., 'var slideshare_mobile_object')]"))
#        slideObj = getElementText(tree.xpath("//script[contains(., 'var slideshare_mobile_object')]")[0])
#        slideid = int(reg_extract(r"slideid:\s(\d+)", slideObj, 1))
#        if slideid == 0:
#            print "This slide doesn't support"
#            return
#        links = []
#        for i in range(1, totalSlides + 1):
#            links.append("{0}-slide-{1}.jpg&big=1".format(baseSlideUrl, i))
#    
    except:
        print sys.exc_info()
        pass
    # -- end func -->
    
def processList():
    # -- Paste list link here --
    urls = '''
    http://www.slideshare.net/misteroo/how-to-market-your-app
	http://www.slideshare.net/startuplessonslearned/dropbox-startup-lessons-learned-2011
	http://www.slideshare.net/pinchmedia/iphone-appstore-secrets-pinch-media
	http://www.slideshare.net/djc1805/iphone-apps-what-how-why
	http://www.slideshare.net/djc1805/2010-digital-trends-ideas-and-technologies-part-1
	http://www.slideshare.net/rudydw/mobile-trends-2020
	http://www.slideshare.net/andreasc/mobile-megatrends-2011-visionmobile
	http://www.slideshare.net/rudydw/mobile-trends-2020-africa
	http://www.slideshare.net/yiibu/beyond-themobilewebbyyiibu
	http://www.slideshare.net/yiibu/adaptation-why-responsive-design-actually-begins-on-the-server
	http://www.slideshare.net/yiibu/pragmatic-responsive-design
	http://www.slideshare.net/yiibu/letting-go-9109114
	http://www.slideshare.net/yiibu/muddling-through-the-mobile-web
	http://www.slideshare.net/pinchmedia/paid-applications-on-the-app-store
	http://www.slideshare.net/pinchmedia/piracy-on-the-appstore
	http://www.slideshare.net/createwithcontext/how-people-really-use-the-iphone-presentation
	http://www.slideshare.net/ginsburgdesign/an-agile-approach-to-iphone-design-paper-prototyping-user-testing
	http://www.slideshare.net/toolmantim/developing-for-the-iphone-presentation
	http://www.slideshare.net/keithahern/how-to-get-ahead-in-the-apple-app-store-by-mogeneration-for-mobile-monday-sydney-04052009
	http://www.slideshare.net/kerryspyder/piracy-vs-music-industry
	http://www.slideshare.net/joshclark/best-iphone-apps-the-musthave-downloads
	http://www.slideshare.net/bess.ho/best-practicei-phone-svcc2009key
	http://www.slideshare.net/fling/mobile-20-design-develop-for-the-iphone-and-beyond
	http://www.slideshare.net/pinchmedia/paid-applications-on-the-app-store
	http://www.slideshare.net/grigs/native-vs-mobile-web-vs-hybrid-apps-for-mobile-development
	http://www.slideshare.net/IC_Agency/i-phone-apps-luxury-brands
	http://www.slideshare.net/ginsburgdesign/an-agile-approach-to-iphone-design-paper-prototyping-user-testing
	http://www.slideshare.net/mobileuserexperience/everything-you-wanted-to-know-about-the-mobile-web-but-were-afraid-to-askblueflavorcom
	http://www.slideshare.net/djc1805/social-marketing-listening-and-engagement
	http://www.slideshare.net/pinchmedia/iphone-appstore-secrets-pinch-media
	http://www.slideshare.net/jeradhill/marketing-tips-for-iphone-app-developers
	
	

	
    '''
    urls = urls.strip().split("\n")
    for url in urls:
        if url==None or url=='': continue
        process(url)
    # -- end func -->
    
if __name__ == '__main__':
    __ROOT__ = '/home/hoangnamhai/HarvestedData/slideshare/'
    processList()
    print "\n\n\n>> Finished ", datetime.datetime.now()
    os._exit(1); os.kill(os.getpid(), 9)
    
